import requests
from lxml import etree
import json

user_agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36"
headers = {
    "User-Agent": user_agent
}


respon = requests.get("https://www.kuaidaili.com/proxylist/2/", headers=headers)

html = etree.HTML(respon.content.decode())
items = html.xpath("//div[@id='freelist']//table[@class='table table-b table-bordered table-striped']//tbody//tr")
# print(items)

ip_info = {}
for item in items:
    ip_info["ip"] = item.xpath("./td[1]/text()")
    # print(ip_info["ip"])
    ip_info["port"] = item.xpath("./td[2]/text()")
    # print(ip_info["port"])
    ip_info["type"] = item.xpath("./td[4]/text()")
    # print(ip_info["type"])
    ip_info["get_post"] = item.xpath("./td[5]/text()")
    # print(ip_info["get_post"])
    with open("./ip.json", "a") as f:
        json.dump(ip_info, f)
